#!/bin/bash

#   Copyright (c) 2014 John Biddiscombe
#   Adapted from stuff found originally somewhere on the internet
#
#   Distributed under the Boost Software License, Version 1.0. (See accompanying
#   file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

# This function writes a slurm script. 
# We can call it with different parameter 
# settings to create different experiments

function write_script
{
JOB_NAME=$(printf 'hpx-N%04d-T%05d-t%02d-%s' ${NODES} ${TRANSFERSIZE} ${THREADS_PERTASK} ${PARCELTYPE})
DIR_NAME=$(printf 'hpx-N%04d-T%05d-t%02d-%s' ${NODES} ${TRANSFERSIZE} ${THREADS_PERTASK} ${PARCELTYPE})
TASKS_PER_NODE=1

echo "Creating job $DIR_NAME"

mkdir -p $DIR_NAME

cat << _EOF_ > ${DIR_NAME}/submit-job.bash
#!/bin/bash

#@ comment = "HPX network-storage"
#@ job_name = ${JOB_NAME}
#@ error = ${DIR_NAME}/output.err
#@ output = ${DIR_NAME}/output.out
#@ environment = COPY_ALL
#@ wall_clock_limit = 01:00:00
#@ notification = never
#@ job_type = bluegene
#@ bg_size = ${NODES}
#@ class = test
#@ queue = ${QUEUE}

export LD_LIBRARY_PATH=${LIB_PATH}:${LD_LIBRARY_PATH}

runjob --ranks-per-node=1 --cwd=$PWD : ${EXECUTABLE1} ${PROGRAM_PARAMS}

_EOF_

chmod 775 ${DIR_NAME}/submit-job.bash

echo "cd ${DIR_NAME}; sbatch submit-job.bash; cd \$BASEDIR" >> run_jobs.bash

}

# get the path to this generate script, works for most cases
pushd `dirname $0` > /dev/null
BASEDIR=`pwd`
popd > /dev/null
echo "Generating jobs using base directory $BASEDIR"

# Create another script to submit all generated jobs to the scheduler
echo "#!/bin/bash" > run_jobs.bash
echo "BASEDIR=$BASEDIR" >> run_jobs.bash
echo "cd $BASEDIR" >> run_jobs.bash
chmod 775 run_jobs.bash

#
# 
#
MPIEXEC="@MPIEXEC@"
QUEUE=nvp0
EXECUTABLE1=@EXE_PATH@
LIB_PATH="@LIB_PATH@"
JOB_OPTIONS1="@JOB_OPTIONS1@"
MEMPERNODE=
TIME="02:00:00"
CLIENTS_PERNODE=0
SERVERS_PERNODE=1

# Loop through all the parameter combinations generating jobs for each
# "tcp" "mpi"
for NODES in 1 2 4 8 16
do
  for PARCELTYPE in "mpi"
  do
    for TRANSFERSIZE in 1024 2048 4096 8192 16384
    do
      for THREADS_PERTASK in 2 4 8 16 32
      do
        LOCAL_SIZE=$(printf "%.0f" $( bc <<< "scale=6;(128 * $TRANSFERSIZE * $THREADS_PERTASK)/1024" ))
        LOCAL_SIZE=$(echo $((LOCAL_SIZE>512?512:LOCAL_SIZE)))
        PROGRAM_PARAMS="--hpx:run-hpx-main -Ihpx.parcel.${PARCELTYPE}.enable=1 --hpx:threads=${THREADS_PERTASK} --hpx:bind=balanced " \
                       "--localMB=${LOCAL_SIZE} --transferKB=${TRANSFERSIZE} --parceltype=${PARCELTYPE} "
        write_script
      done
    done
  done
done

echo "echo \"Use find . -name \*.out -exec grep CSVData {} \;\" " >> run_jobs.bash
